{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "cd9da1a3",
   "metadata": {},
   "source": [
    "# 请求数据准备"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d796974c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# header\n",
    "import liepindata\n",
    "headers = liepindata.headers()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "28bf122f",
   "metadata": {},
   "source": [
    "* 备注\n",
    "\n",
    "> 1.城市：'全国'、'北京'、'上海'...\n",
    "\n",
    "> 2.关键词：不限制\n",
    "\n",
    "> 3.工作经验：'应届生'、'实习生'、'1年以内'...\n",
    "\n",
    "> 4.公司规模: '1-49人'、'50-99人'、'100-499人'..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "2592b93c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'data': {'mainSearchPcConditionForm': {'city': '050090',\n",
       "   'dq': '050090',\n",
       "   'pubTime': '',\n",
       "   'currentPage': '0',\n",
       "   'pageSize': 40,\n",
       "   'key': '数据分析师',\n",
       "   'suggestTag': '',\n",
       "   'workYearCode': '2',\n",
       "   'compId': '',\n",
       "   'compName': '',\n",
       "   'compTag': '',\n",
       "   'industry': '',\n",
       "   'salary': '',\n",
       "   'jobKind': '',\n",
       "   'compScale': '010',\n",
       "   'compKind': '',\n",
       "   'compStage': '',\n",
       "   'eduLevel': ''},\n",
       "  'passThroughForm': {'scene': 'condition',\n",
       "   'skId': 'geq5b0qsi9umzbhn0oojbi2j8v8qu2bb',\n",
       "   'fkId': 'geq5b0qsi9umzbhn0oojbi2j8v8qu2bb',\n",
       "   'ckId': 'y2jy1uvl2gar8xipyemg77rijoomop78',\n",
       "   'suggest': None}}}"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# payload\n",
    "payload = liepindata.request_payload(城市=\"深圳\",关键词=\"数据分析师\",工作经验=\"实习生\",公司规模=\"1-49人\")\n",
    "payload"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "465cf4bb",
   "metadata": {},
   "source": [
    "# 指定数据抓取"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e4c466bc",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "from requests_html import HTMLSession\n",
    "\n",
    "session = HTMLSession()\n",
    "r = session.post(\n",
    "            url = 'https://api-c.liepin.com/api/com.liepin.searchfront4c.pc-search-job',\n",
    "            data = json.dumps(payload), # POST请求带的参数\n",
    "            headers = headers    # 模拟真实用户\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "db62dc67",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'flag': 1,\n",
       " 'data': {'data': {'jobCardList': [{'dataInfo': '%7B%22pageSize%22%3A%2240%22%2C%22curPage%22%3A%220%22%2C%22jobId%22%3A%2262071461%22%2C%22jobKind%22%3A%226%22%2C%22sfrom%22%3A%22search_job_pc%22%2C%22ckId%22%3A%22y2jy1uvl2gar8xipyemg77rijoomop78%22%2C%22scene%22%3A%22condition%22%2C%22skId%22%3A%22geq5b0qsi9umzbhn0oojbi2j8v8qu2bb%22%2C%22fkId%22%3A%22y2jy1uvl2gar8xipyemg77rijoomop78%22%2C%22d_curPage%22%3A%220%22%2C%22d_pageSize%22%3A%2240%22%2C%22as_from%22%3A%22search_prime%22%7D',\n",
       "     'job': {'labels': ['1个月', '本科'],\n",
       "      'title': '电商数据分析师',\n",
       "      'refreshTime': '20231009152530',\n",
       "      'jobId': '62071461',\n",
       "      'jobKind': '6',\n",
       "      'dq': '深圳-龙华区',\n",
       "      'topJob': False,\n",
       "      'pcOuterLink': '',\n",
       "      'h5OuterLink': '',\n",
       "      'salary': '100-200元/天',\n",
       "      'link': 'https://www.liepin.com/lptjob/62071461',\n",
       "      'dataPromId': 'd_sfrom=search_prime&d_ckId=null&d_curPage=0&d_pageSize=40&d_headId=null&d_posi=0&skId=geq5b0qsi9umzbhn0oojbi2j8v8qu2bb&fkId=y2jy1uvl2gar8xipyemg77rijoomop78&ckId=y2jy1uvl2gar8xipyemg77rijoomop78&sfrom=search_job_pc&curPage=0&pageSize=40&index=0',\n",
       "      'campusJobKind': '实习',\n",
       "      'advViewFlag': False},\n",
       "     'recruiter': {'recruiterName': '戴先生',\n",
       "      'recruiterTitle': '部门主管',\n",
       "      'chatted': False,\n",
       "      'imId': '9792e818d4dc8f43b8186adbf6c31dd6',\n",
       "      'imUserType': '2',\n",
       "      'recruiterId': 'b06533f5e3a36c6bf95cd792e49c0cb3',\n",
       "      'recruiterPhoto': '5f8f986bdfb13a7dee342f2108u.jpg'},\n",
       "     'comp': {'compLogo': '64dd9438f5796c30a18b3e4308u.jpg',\n",
       "      'compName': '趣宠儿',\n",
       "      'compScale': '1-49人',\n",
       "      'compId': 13646711,\n",
       "      'compIndustry': '批发/零售',\n",
       "      'link': 'https://www.liepin.com/company/13646711/'},\n",
       "     'dataParams': '{\"recruiterName\":\"戴先生\",\"jobId\":\"62071461\",\"jobKind\":\"6\",\"imId\":\"9792e818d4dc8f43b8186adbf6c31dd6\",\"imUserType\":\"2\",\"userId\":\"b06533f5e3a36c6bf95cd792e49c0cb3\",\"recruiterPhoto\":\"5f8f986bdfb13a7dee342f2108u.jpg\"}'},\n",
       "    {'dataInfo': '%7B%22pageSize%22%3A%2240%22%2C%22curPage%22%3A%220%22%2C%22jobId%22%3A%2256713457%22%2C%22jobKind%22%3A%226%22%2C%22sfrom%22%3A%22search_job_pc%22%2C%22ckId%22%3A%22y2jy1uvl2gar8xipyemg77rijoomop78%22%2C%22scene%22%3A%22condition%22%2C%22skId%22%3A%22geq5b0qsi9umzbhn0oojbi2j8v8qu2bb%22%2C%22fkId%22%3A%22y2jy1uvl2gar8xipyemg77rijoomop78%22%2C%22d_curPage%22%3A%220%22%2C%22d_pageSize%22%3A%2240%22%2C%22as_from%22%3A%22search_prime%22%7D',\n",
       "     'job': {'labels': ['3个月', '提供转正', '本科', '数据建模', 'Pandas'],\n",
       "      'title': '前台量化分析师 Desk Quant（实习）',\n",
       "      'refreshTime': '20230327091836',\n",
       "      'jobId': '56713457',\n",
       "      'jobKind': '6',\n",
       "      'dq': '深圳-福田区',\n",
       "      'topJob': False,\n",
       "      'pcOuterLink': '',\n",
       "      'h5OuterLink': '',\n",
       "      'salary': '200-400元/天',\n",
       "      'link': 'https://www.liepin.com/lptjob/56713457',\n",
       "      'dataPromId': 'd_sfrom=search_prime&d_ckId=null&d_curPage=0&d_pageSize=40&d_headId=null&d_posi=1&skId=geq5b0qsi9umzbhn0oojbi2j8v8qu2bb&fkId=y2jy1uvl2gar8xipyemg77rijoomop78&ckId=y2jy1uvl2gar8xipyemg77rijoomop78&sfrom=search_job_pc&curPage=0&pageSize=40&index=1',\n",
       "      'campusJobKind': '实习',\n",
       "      'advViewFlag': False},\n",
       "     'recruiter': {'recruiterName': '陈女士',\n",
       "      'recruiterTitle': 'HR',\n",
       "      'chatted': False,\n",
       "      'imId': '2adfa4d59d1f3efe39e48140c0193d97',\n",
       "      'imUserType': '2',\n",
       "      'recruiterId': '4b03d5cd244cc46345c190c582ffc5aa',\n",
       "      'recruiterPhoto': '5f8f98648dbe6273dcf8515508u.jpg'},\n",
       "     'comp': {'compLogo': '64129a0e416e883aa1c773b303u.png',\n",
       "      'compName': '盖亚青柯',\n",
       "      'compScale': '1-49人',\n",
       "      'compId': 13353691,\n",
       "      'compIndustry': '基金/证券/期货',\n",
       "      'link': 'https://www.liepin.com/company/13353691/'},\n",
       "     'dataParams': '{\"recruiterName\":\"陈女士\",\"jobId\":\"56713457\",\"jobKind\":\"6\",\"imId\":\"2adfa4d59d1f3efe39e48140c0193d97\",\"imUserType\":\"2\",\"userId\":\"4b03d5cd244cc46345c190c582ffc5aa\",\"recruiterPhoto\":\"5f8f98648dbe6273dcf8515508u.jpg\"}'},\n",
       "    {'dataInfo': '%7B%22pageSize%22%3A%2240%22%2C%22curPage%22%3A%220%22%2C%22jobId%22%3A%2256308363%22%2C%22jobKind%22%3A%226%22%2C%22sfrom%22%3A%22search_job_pc%22%2C%22ckId%22%3A%22y2jy1uvl2gar8xipyemg77rijoomop78%22%2C%22scene%22%3A%22condition%22%2C%22skId%22%3A%22geq5b0qsi9umzbhn0oojbi2j8v8qu2bb%22%2C%22fkId%22%3A%22y2jy1uvl2gar8xipyemg77rijoomop78%22%2C%22d_curPage%22%3A%220%22%2C%22d_pageSize%22%3A%2240%22%2C%22as_from%22%3A%22search_prime%22%7D',\n",
       "     'job': {'labels': ['5个月', '提供转正', '本科'],\n",
       "      'title': '行业数据分析实习生',\n",
       "      'refreshTime': '20231010144842',\n",
       "      'jobId': '56308363',\n",
       "      'jobKind': '6',\n",
       "      'dq': '深圳-福田区',\n",
       "      'topJob': False,\n",
       "      'pcOuterLink': '',\n",
       "      'h5OuterLink': '',\n",
       "      'salary': '100-150元/天',\n",
       "      'link': 'https://www.liepin.com/lptjob/56308363',\n",
       "      'dataPromId': 'd_sfrom=search_prime&d_ckId=null&d_curPage=0&d_pageSize=40&d_headId=null&d_posi=2&skId=geq5b0qsi9umzbhn0oojbi2j8v8qu2bb&fkId=y2jy1uvl2gar8xipyemg77rijoomop78&ckId=y2jy1uvl2gar8xipyemg77rijoomop78&sfrom=search_job_pc&curPage=0&pageSize=40&index=2',\n",
       "      'campusJobKind': '实习',\n",
       "      'advViewFlag': False},\n",
       "     'recruiter': {'recruiterName': '张女士',\n",
       "      'recruiterTitle': '人力资源部专员',\n",
       "      'chatted': False,\n",
       "      'imId': '5ce1484d9a86ef3771de52c3bc0b6279',\n",
       "      'imUserType': '2',\n",
       "      'recruiterId': '903b24856228e8430412b4d953179852',\n",
       "      'recruiterPhoto': '5febe70f3059cc17bf2e6ec407u.png'},\n",
       "     'comp': {'compLogo': '63ecb24a7e3d5966c13512e803u.jpg',\n",
       "      'compName': '合丰投(深圳)投资有限公司',\n",
       "      'compScale': '1-49人',\n",
       "      'compId': 12517119,\n",
       "      'compIndustry': '基金/证券/期货',\n",
       "      'link': 'https://www.liepin.com/company/12517119/'},\n",
       "     'dataParams': '{\"recruiterName\":\"张女士\",\"jobId\":\"56308363\",\"jobKind\":\"6\",\"imId\":\"5ce1484d9a86ef3771de52c3bc0b6279\",\"imUserType\":\"2\",\"userId\":\"903b24856228e8430412b4d953179852\",\"recruiterPhoto\":\"5febe70f3059cc17bf2e6ec407u.png\"}'}],\n",
       "   'compList': []},\n",
       "  'pagination': {'pageSize': 40,\n",
       "   'currentPage': 0,\n",
       "   'totalPage': 1,\n",
       "   'totalCounts': 3,\n",
       "   'hasNext': False},\n",
       "  'passThroughData': {'sfrom': 'search_job_pc',\n",
       "   'ckId': 'y2jy1uvl2gar8xipyemg77rijoomop78',\n",
       "   'scene': 'condition',\n",
       "   'skId': 'geq5b0qsi9umzbhn0oojbi2j8v8qu2bb',\n",
       "   'fkId': 'y2jy1uvl2gar8xipyemg77rijoomop78'}}}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "liepin_data = r.json()\n",
    "liepin_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "4ef3ef8f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dataInfo</th>\n",
       "      <th>dataParams</th>\n",
       "      <th>job.labels</th>\n",
       "      <th>job.title</th>\n",
       "      <th>job.refreshTime</th>\n",
       "      <th>job.jobId</th>\n",
       "      <th>job.jobKind</th>\n",
       "      <th>job.dq</th>\n",
       "      <th>job.topJob</th>\n",
       "      <th>job.pcOuterLink</th>\n",
       "      <th>...</th>\n",
       "      <th>recruiter.imId</th>\n",
       "      <th>recruiter.imUserType</th>\n",
       "      <th>recruiter.recruiterId</th>\n",
       "      <th>recruiter.recruiterPhoto</th>\n",
       "      <th>comp.compLogo</th>\n",
       "      <th>comp.compName</th>\n",
       "      <th>comp.compScale</th>\n",
       "      <th>comp.compId</th>\n",
       "      <th>comp.compIndustry</th>\n",
       "      <th>comp.link</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>%7B%22pageSize%22%3A%2240%22%2C%22curPage%22%3...</td>\n",
       "      <td>{\"recruiterName\":\"戴先生\",\"jobId\":\"62071461\",\"job...</td>\n",
       "      <td>[1个月, 本科]</td>\n",
       "      <td>电商数据分析师</td>\n",
       "      <td>20231009152530</td>\n",
       "      <td>62071461</td>\n",
       "      <td>6</td>\n",
       "      <td>深圳-龙华区</td>\n",
       "      <td>False</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>9792e818d4dc8f43b8186adbf6c31dd6</td>\n",
       "      <td>2</td>\n",
       "      <td>b06533f5e3a36c6bf95cd792e49c0cb3</td>\n",
       "      <td>5f8f986bdfb13a7dee342f2108u.jpg</td>\n",
       "      <td>64dd9438f5796c30a18b3e4308u.jpg</td>\n",
       "      <td>趣宠儿</td>\n",
       "      <td>1-49人</td>\n",
       "      <td>13646711</td>\n",
       "      <td>批发/零售</td>\n",
       "      <td>https://www.liepin.com/company/13646711/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>%7B%22pageSize%22%3A%2240%22%2C%22curPage%22%3...</td>\n",
       "      <td>{\"recruiterName\":\"陈女士\",\"jobId\":\"56713457\",\"job...</td>\n",
       "      <td>[3个月, 提供转正, 本科, 数据建模, Pandas]</td>\n",
       "      <td>前台量化分析师 Desk Quant（实习）</td>\n",
       "      <td>20230327091836</td>\n",
       "      <td>56713457</td>\n",
       "      <td>6</td>\n",
       "      <td>深圳-福田区</td>\n",
       "      <td>False</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>2adfa4d59d1f3efe39e48140c0193d97</td>\n",
       "      <td>2</td>\n",
       "      <td>4b03d5cd244cc46345c190c582ffc5aa</td>\n",
       "      <td>5f8f98648dbe6273dcf8515508u.jpg</td>\n",
       "      <td>64129a0e416e883aa1c773b303u.png</td>\n",
       "      <td>盖亚青柯</td>\n",
       "      <td>1-49人</td>\n",
       "      <td>13353691</td>\n",
       "      <td>基金/证券/期货</td>\n",
       "      <td>https://www.liepin.com/company/13353691/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>%7B%22pageSize%22%3A%2240%22%2C%22curPage%22%3...</td>\n",
       "      <td>{\"recruiterName\":\"张女士\",\"jobId\":\"56308363\",\"job...</td>\n",
       "      <td>[5个月, 提供转正, 本科]</td>\n",
       "      <td>行业数据分析实习生</td>\n",
       "      <td>20231010144842</td>\n",
       "      <td>56308363</td>\n",
       "      <td>6</td>\n",
       "      <td>深圳-福田区</td>\n",
       "      <td>False</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>5ce1484d9a86ef3771de52c3bc0b6279</td>\n",
       "      <td>2</td>\n",
       "      <td>903b24856228e8430412b4d953179852</td>\n",
       "      <td>5febe70f3059cc17bf2e6ec407u.png</td>\n",
       "      <td>63ecb24a7e3d5966c13512e803u.jpg</td>\n",
       "      <td>合丰投(深圳)投资有限公司</td>\n",
       "      <td>1-49人</td>\n",
       "      <td>12517119</td>\n",
       "      <td>基金/证券/期货</td>\n",
       "      <td>https://www.liepin.com/company/12517119/</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 29 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            dataInfo  \\\n",
       "0  %7B%22pageSize%22%3A%2240%22%2C%22curPage%22%3...   \n",
       "1  %7B%22pageSize%22%3A%2240%22%2C%22curPage%22%3...   \n",
       "2  %7B%22pageSize%22%3A%2240%22%2C%22curPage%22%3...   \n",
       "\n",
       "                                          dataParams  \\\n",
       "0  {\"recruiterName\":\"戴先生\",\"jobId\":\"62071461\",\"job...   \n",
       "1  {\"recruiterName\":\"陈女士\",\"jobId\":\"56713457\",\"job...   \n",
       "2  {\"recruiterName\":\"张女士\",\"jobId\":\"56308363\",\"job...   \n",
       "\n",
       "                      job.labels               job.title job.refreshTime  \\\n",
       "0                      [1个月, 本科]                 电商数据分析师  20231009152530   \n",
       "1  [3个月, 提供转正, 本科, 数据建模, Pandas]  前台量化分析师 Desk Quant（实习）  20230327091836   \n",
       "2                [5个月, 提供转正, 本科]               行业数据分析实习生  20231010144842   \n",
       "\n",
       "  job.jobId job.jobKind  job.dq  job.topJob job.pcOuterLink  ...  \\\n",
       "0  62071461           6  深圳-龙华区       False                  ...   \n",
       "1  56713457           6  深圳-福田区       False                  ...   \n",
       "2  56308363           6  深圳-福田区       False                  ...   \n",
       "\n",
       "                     recruiter.imId recruiter.imUserType  \\\n",
       "0  9792e818d4dc8f43b8186adbf6c31dd6                    2   \n",
       "1  2adfa4d59d1f3efe39e48140c0193d97                    2   \n",
       "2  5ce1484d9a86ef3771de52c3bc0b6279                    2   \n",
       "\n",
       "              recruiter.recruiterId         recruiter.recruiterPhoto  \\\n",
       "0  b06533f5e3a36c6bf95cd792e49c0cb3  5f8f986bdfb13a7dee342f2108u.jpg   \n",
       "1  4b03d5cd244cc46345c190c582ffc5aa  5f8f98648dbe6273dcf8515508u.jpg   \n",
       "2  903b24856228e8430412b4d953179852  5febe70f3059cc17bf2e6ec407u.png   \n",
       "\n",
       "                     comp.compLogo  comp.compName comp.compScale comp.compId  \\\n",
       "0  64dd9438f5796c30a18b3e4308u.jpg            趣宠儿          1-49人    13646711   \n",
       "1  64129a0e416e883aa1c773b303u.png           盖亚青柯          1-49人    13353691   \n",
       "2  63ecb24a7e3d5966c13512e803u.jpg  合丰投(深圳)投资有限公司          1-49人    12517119   \n",
       "\n",
       "   comp.compIndustry                                 comp.link  \n",
       "0              批发/零售  https://www.liepin.com/company/13646711/  \n",
       "1           基金/证券/期货  https://www.liepin.com/company/13353691/  \n",
       "2           基金/证券/期货  https://www.liepin.com/company/12517119/  \n",
       "\n",
       "[3 rows x 29 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "pd.json_normalize(liepin_data['data']['data']['jobCardList'])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
